/*
 * Copyright (c) 2008-2009, Computational Crawling LP
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 *
 *    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 *    * Neither the name of Computational Crawling LP, 80legs, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package com.eightylegs.customer.sample;

import java.util.Collection;
import org.apache.commons.lang.StringEscapeUtils;

import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.eightylegs.customer.Default80AppPropertyKeys;
import com.eightylegs.customer.I80App;

public class Sample80App implements I80App {
	

	public String getVersion() {
		return "80App_1.2";  // this must be set correctly or 80legs will not use your jar.
	}
	

	public void initialize( Properties properties, byte[] data ) {
	 
		 
	}
	
	public Collection<String> parseLinks(byte[] documentContent, String url, Map<String, String> headers, Map<Default80AppPropertyKeys, Object> default80AppProperties, String statusCodeLine) {
		return null;  // returning null means that 80legs will use its own default parseLinks()
	}
	
	public byte[] processDocument(byte[] documentContent, String url, Map<String, String> headers, Map<Default80AppPropertyKeys, Object> default80AppProperties, String statusCodeLine) {

		String result = "";

		try {
			String charset = "UTF-8";
			String pageContents = byteArrayToString(documentContent, 0, documentContent.length, charset);
			Pattern p1 = Pattern.compile("<meta .*charset=.*\".*/>",Pattern.CASE_INSENSITIVE);
			Matcher m1 = p1.matcher(pageContents);
			if (m1.find()) {
				Pattern p2 = Pattern.compile("charset=([a-z]|-|[0-9])*",Pattern.CASE_INSENSITIVE);
				Matcher m2 = p2.matcher(m1.group());
				if (m2.find()) {
					charset = m2.group().substring(8);
				}
			}
			
			pageContents = byteArrayToString(documentContent, 0, documentContent.length, charset);
			pageContents = StringEscapeUtils.unescapeHtml(pageContents);
			
			String[] r_remove = {
					"<\\?xml[\\s\\S]*?>",
					"<script[\\s\\S]*?</script>",
					"<style[\\s\\S]*?</style>",
					"<.*?>",
					"<![\\s\\S]*?>",
					"(\\n|\\r|\\t|,|\\.|\\!|\\?|\")",
					"\\s{1,}"
			};

			for (int i = 0; i < r_remove.length; i++) {
				Pattern p = Pattern.compile(r_remove[i], Pattern.CASE_INSENSITIVE);
				Matcher m = p.matcher(pageContents);
				while (m.find()) {
					pageContents = pageContents.replace(m.group(), " ");
				}
			}
			
			String[] keywords = pageContents.split(" ");
			for (int i = 0; i < keywords.length; i++) {
				keywords[i] = keywords[i].replaceAll("[\\+<>=\\.,\\)\\(:;]\\|&", "").toLowerCase();
			}
			HashMap<String, Integer> unique_keywords = new HashMap<String, Integer>();
			
			for (int i = 0; i < keywords.length; i++) {
				if (!unique_keywords.containsKey(keywords[i])) {
					unique_keywords.put(keywords[i], 1);
				} else {
					unique_keywords.put(keywords[i], unique_keywords.get(keywords[i]) + 1);
				}
			}
			unique_keywords.remove("");
			unique_keywords.remove(" ");
			
			Iterator<String> itr = unique_keywords.keySet().iterator();
			while (itr.hasNext()) {
				String key = itr.next().toString();
				Integer value = unique_keywords.get(key);
				result = result + key + "\t" + value.toString() + "\n";
			}

			return result.getBytes(charset);
			
		} catch (Exception e) {
			e.printStackTrace();
			return null;
		}

	}
	
	
	
	
	 	// EXAMPLE CODE: This method is used in the 
	 	// example code above.
	 	 
		public static String byteArrayToString(byte[] b, int byteStart, int numBytes, String charset) throws Exception
		{
			return new String (b, byteStart, numBytes, charset);
		}
	
		
}
